//抓取链家房价
package main

import (
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = "lianjia"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	//城市编号
	var citycode = flag.String("citycode", "bj", "请输入城市编号!")
	var curpage = flag.Int("curpage", 0, "请输入开始page!")
	flag.Parse()

	switch *citycode {
	default:
		fallthrough
	case "bj":
		{
			main_domain = "http://bj.lianjia.com"
			LOG = "bj." + LOG
			break
		}
	case "tj":
		{
			main_domain = "http://tj.lianjia.com"
			LOG = "tj." + LOG
			break
		}
	case "sh":
		{
			main_domain = "http://sh.lianjia.com"
			LOG = "sh." + LOG
			break
		}
	}
	url := main_domain + "/xiaoqu/"
	getPages(url, *curpage)
	fmt.Println("done.")
}

//获取分布
func getPages(url string, curpage int) {
	if curpage > 0 {
		tmpurl := fmt.Sprintf("%spg%d/", url, curpage)
		waitgroup.Add(1)
		go getContent(tmpurl, curpage)
		waitgroup.Wait()
		return
	}
	doc, e := getDoc(url)
	if e != nil {
		s(url, e.Error())
	} else {
		pagelink, ok := doc.Find("a:contains('尾页')").Attr("href")
		if ok {
			pagelink = strings.Replace(pagelink, url, "", -1)
			pagelink = strings.Replace(pagelink, "/", "", -1)
			pagelink = strings.Replace(pagelink, "pg", "", -1)
			pageCount, e := strconv.Atoi(pagelink)
			if e != nil {
				s("没有找到总页数")
			} else {
				str := "page|"
				str += "百度坐标lng" + "|"
				str += "百度坐标lat" + "|"
				str += "楼盘名称" + "|"
				str += "热点区县" + "|"
				str += "热点板块" + "|"
				str += "距离地铁站" + "|"
				str += "环比上月(涨跌幅)" + "|"
				str += "成交均价元/平米" + "|"
				str += "近30天成交套数"
				outlog(str)
				for i := 1; i <= pageCount; i += 10 {
					for j := i; j < i+10; j++ {
						tmpurl := fmt.Sprintf("%spg%d/", url, j)
						waitgroup.Add(1)
						go getContent(tmpurl, j)
					}
					waitgroup.Wait()
					time.Sleep(time.Second * 10)
				}
			}
		}
	}
}

//获取内容
func getContent(url string, page int) {
	doc, e := getDoc(url)
	if e != nil {
		s(url, e.Error())
	} else {
		outArr := []map[string]string{}
		doc.Find("#listData").Children().Each(func(i int, sec *goq.Selection) {
			/*
				lng           string //百度坐标lng
				lat           string //百度坐标lat
				communityname string //楼盘名称
				district      string //热点区县
				block         string //热点板块
				shenlv        string //距离地铁站
				price         string //环比上月,涨跌幅
				priceoo       string //成交均价
				priceooo      string //近30天成交
			*/
			newmap := map[string]string{}

			yldt, ok := sec.Find(".yldt").Find("a").Attr("onclick")
			if ok {
				yldt = strings.Replace(yldt, "ditu(", "", -1)
				yldt = strings.Replace(yldt, ")", "", -1)
				yldt = strings.Replace(yldt, "'", "", -1)
				yldt = strings.Replace(yldt, " ", "", -1)
				maparr := strings.Split(yldt, ",")
				if len(maparr) > 5 {
					newmap["lng"] = maparr[0]
					newmap["lat"] = maparr[1]
					newmap["communityname"] = maparr[2]
					newmap["district"] = maparr[3]
					newmap["block"] = maparr[4]
				}
			}

			shenlv := sec.Find(".shenlv").Text()
			newmap["shenlv"] = shenlv

			sec.Find(".price").Find("li").Each(func(i int, sec *goq.Selection) {
				newmap["price"] += sec.Text()
			})

			newmap["priceoo"] = sec.Find(".priceoo").Find("span").Text()
			newmap["priceooo"] = sec.Find(".priceooo").Find("kbd").Text()

			if newmap["communityname"] != "" && newmap["priceoo"] != "" {
				outArr = append(outArr, newmap)
			}
		})

		for _, v := range outArr {
			str := fmt.Sprintf("[%d]", page) + "|"
			str += filterStr(v["lng"]) + "|"
			str += filterStr(v["lat"]) + "|"
			str += filterStr(v["communityname"]) + "|"
			str += filterStr(v["district"]) + "|"
			str += filterStr(v["block"]) + "|"
			str += filterStr(v["shenlv"]) + "|"
			str += filterStr(v["price"]) + "|"
			str += filterStr(v["priceoo"]) + "|"
			str += filterStr(v["priceooo"])
			outlog(str)
		}
	}
	waitgroup.Done()
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
